#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Nov 29 19:11:50 2018

@author: elsa
"""

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Nov 27 16:07:13 2018

@author: elsa
"""

import socket
import socks
import requests
import os
from bs4 import BeautifulSoup
import time
import pandas as pd

headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36'
}

socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", 1080)
socket.socket = socks.socksocket

name_data   = []
phone_data  = []
wechat_data = []
#whatsapp_data = []  与电话合并
urls_data = []
video_data =[]

#每个网站需要单独解析，因为目录结构不同
#每个网站的详情也需要单独解析，用的字段有点区别

#1
'''
url='http://sgrelax.org/'
r = requests.get(url, headers=headers)
r.raise_for_status()    # 如果响应状态码不是 200，就主动抛出异常
#<h3 class="product-name"><a href="http://sgrelax.org/girl/112-xiaoxiao/">112号 &#8211; Xiao Xiao 小小</a></h3>
soup = BeautifulSoup(r.text, 'html5lib')  # html5lib为pythonista上的解析器

try:
    for people in soup.find_all('h3'):
        purl  = people.find('a')['href']
        pname = people.find('a').text
        print (purl,pname)
        #另一个页面，电话与微信
        time.sleep( 1)
        urls_data.append(purl)
        rp = requests.get(purl, headers=headers)
        rp.raise_for_status()    # 如果响应状态码不是 200，就主动抛出异常
        psoup = BeautifulSoup(rp.text, 'html5lib')  # html5lib为pythonista上的解析器
        #<div class="att_label">
	        #Mobile:					</div>
			#		<div class="att_value">
			#			<p>8154 7118</p>
                        
        pphone = ''
        pwechat= ''
        pvideo = ''
        
        divs = psoup.find_all('div','col')
        for div in divs:
            att_label = div.find('div', class_="att_label").text
            att_value = div.find('div', class_="att_value").text
            
            if ('Mobile' in att_label):
                pphone = att_value
                
            if ('WeChat' in att_label):
                pwechat = att_value;
       
        name_data.append(pname)    
        
        pphone=pphone.replace(' ','')
        pphone=pphone.strip()
        if (len(pphone)>6):
            pphone = '+65'+pphone
        phone_data.append(pphone)  
        
        wechat_data.append(pwechat)  
        video_data.append(pvideo)

except (AttributeError,TypeError):
    pass
        
time.sleep( 60)
'''
#-------------------------------------------------------------------------------------------

url='https://sex727a.net/'

r = requests.get(url, headers=headers)
r.raise_for_status() 
soup = BeautifulSoup(r.text, 'html5lib')  # html5lib为pythonista上的解析器

try:
    for people in soup.find_all('div','profile_quad'):
        purl  = people.find('a')['href']
        pname = people.find('a').text
        
        purl=url+purl.replace("\n","")
        
        print (purl)
        #另一个页面，电话与微信
        time.sleep( 1)
        urls_data.append(purl)
        rp = requests.get(purl, headers=headers)
        rp.raise_for_status()    # 如果响应状态码不是 200，就主动抛出异常
        psoup = BeautifulSoup(rp.text, 'html5lib')  # html5lib为pythonista上的解析器
        #<span style='color: #00f;'><strong>Mobile 联络号码</strong> : 8446 8249</span><br />           
        pphone = ''
        pwechat= ''
        pvideo = ''
        
        divs = psoup.find_all('span')
        
        for div in divs:
            #att_label = div.find('strong')
            att_value = div.text
           
            if ('Mobile' in att_value):
                nindex = att_value.find(':')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pphone = att_value[nindex:] #截取第七个字符到结尾
                
            if ('Wechat' in att_value):
                pwechat = att_value.find(':')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pwechat = att_value[nindex:] #截取第七个字符到结尾
       
        pname=pname.replace(' ','')
        pname=pname.strip()
        name_data.append(pname)    
        
        pphone=pphone.replace(' ','')
        pphone=pphone.strip()
        if (len(pphone)>6):
            pphone = '+65'+pphone
        phone_data.append(pphone)  
        
        wechat_data.append(pwechat)  
        
        try:
            pvideo = psoup.find('source')['data-src']
            if (len(pvideo)>0):
                pvideo = url+pvideo
        except (AttributeError,TypeError):
            pass
        
        video_data.append(pvideo)
         
        print (pname,pphone,pwechat,pvideo)
        
except (AttributeError,TypeError):
    pass


time.sleep( 60)
print(len(urls_data),len(name_data),len(phone_data),len(wechat_data),len(video_data))
#--------------------------------------------------------------------------------------------

url='http://www.tsrjooxx.net/'

r = requests.get(url, headers=headers)
r.raise_for_status() 
soup = BeautifulSoup(r.text, 'html5lib')  # html5lib为pythonista上的解析器
try:
    
    peoples = soup.find('div','index-rbox').find_all('li')
        
    for people in peoples:
        purl  = people.find('a')['href']  
        purl=url+purl.replace("\n","")
      
        #另一个页面，电话与微信
        time.sleep( 1)
        urls_data.append(purl)
        rp = requests.get(purl, headers=headers)
        rp.raise_for_status()    # 如果响应状态码不是 200，就主动抛出异常
        psoup = BeautifulSoup(rp.text, 'html5lib')  # html5lib为pythonista上的解析器
        #<p><label>Mobile 电话：</label><big>90382241</big></p>	<p><label>WeChat 微信：</label><big>meixi112528</big>
           
        pname = ''
        pphone = ''
        pwechat= ''
        pvideo = ''
        
        about = psoup.find('div','about')
        divs  = psoup.find_all('p')
     
        for div in divs:
           
            att_value = div.text
               
            if ('Name' in att_value):
                nindex = att_value.find(':')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pname = att_value[nindex:] #截取第七个字符到结尾
                
            if ('Mobile' in att_value):
                nindex = att_value.find('：')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pphone = att_value[nindex:] #截取第七个字符到结尾
                
            if ('WeChat' in att_value):
                nindex = att_value.find('：')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pwechat = att_value[nindex:] #截取第七个字符到结尾
       
        pname=pname.replace(' ','')
        pname=pname.strip()
        name_data.append(pname)    
        
        pphone=pphone.replace(' ','')
        pphone=pphone.strip()
        if (len(pphone)>6):
            pphone = '+65'+pphone
        phone_data.append(pphone)    
        
        wechat_data.append(pwechat)  
        
        try:
            pvideo = psoup.find('video')['src']
            if (len(pvideo)>0):
                pvideo = 'http://www.tsrjooxx.net'+pvideo
        except (AttributeError,TypeError):
            pass
        
        video_data.append(pvideo)
        
        print (pname,pphone,pwechat)
        
except (AttributeError,TypeError):
    pass
 
time.sleep( 60)
print(len(urls_data),len(name_data),len(phone_data),len(wechat_data),len(video_data))
#--------------------------------------------------------------------------------
'''
url='http://www.sexwolves.net'

r = requests.get(url, headers=headers)
r.raise_for_status() 
soup = BeautifulSoup(r.text, 'html5lib')  # html5lib为pythonista上的解析器

try:
    
    peoples = soup.find_all('div','box')
   
    for people in peoples:
        purl  = people.find('a')['href']  
        purl=url+purl.replace("\n","")
       
        time.sleep( 1)
        #另一个页面，电话与微信
        urls_data.append(purl)
        rp = requests.get(purl, headers=headers)
        rp.raise_for_status()    # 如果响应状态码不是 200，就主动抛出异常
        psoup = BeautifulSoup(rp.text, 'html5lib')  # html5lib为pythonista上的解析器
        #<p><label>Mobile 电话：</label><big>90382241</big></p>	<p><label>WeChat 微信：</label><big>meixi112528</big>
           
        pname = ''
        pphone = ''
        pwechat= ''
        pvideo = ''
        
        about = psoup.find('div','girl-con')
        divs  = psoup.find_all('p')
  
        for div in divs:
           
            att_value = div.text
           
            if ('Name' in att_value):
                nindex = att_value.find('：')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pname = att_value[nindex:] #截取第七个字符到结尾
                
            if ('Mobile' in att_value):
                nindex = att_value.find(':')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pphone = att_value[nindex:] #截取第七个字符到结尾
                
            if ('Wechat' in att_value):
                nindex = att_value.find(':')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pwechat = att_value[nindex:] #截取第七个字符到结尾
                
            if ('Whatsapp' in att_value):
                nindex = att_value.find(':')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pphone = att_value[nindex:] #截取第七个字符到结尾
       
        pname=pname.replace(' ','')
        pname=pname.strip()
        name_data.append(pname)    
        
        pphone=pphone.replace(' ','')
        pphone=pphone.strip()
        if (len(pphone)>6):
            pphone = '+65'+pphone
        phone_data.append(pphone)    
        
        wechat_data.append(pwechat)  
        video_data.append(pvideo)
        print (pname,pphone,pwechat)
        
except (AttributeError,TypeError):
    pass
    
table=pd.DataFrame({'name':name_data,'phone':phone_data,'wechat':wechat_data,'url':urls_data})

date = time.strftime('%Y%m%d',time.localtime(time.time()))

table.to_csv(r'sg/sexwolves%s.csv' % (date),encoding = "utf_8_sig")

name_data.clear()
phone_data.clear()
wechat_data.clear()
whatsapp_data.clear()
urls_data.clear()
time.sleep( 60)
print(len(urls_data),len(name_data),len(phone_data),len(wechat_data))
'''        
#------------------------------------------------------------------------------------
url='http://www.sgbense.com'

r = requests.get(url, headers=headers)
r.raise_for_status() 
soup = BeautifulSoup(r.text, 'html5lib')  # html5lib为pythonista上的解析器

try:
    
    peoples = soup.find_all('div','girls-item')
   
    for people in peoples:
        purl  = people.find('a')['href']  
        purl=url+purl.replace("\n","")
       
        time.sleep( 1)
        #另一个页面，电话与微信
        urls_data.append(purl)
        rp = requests.get(purl, headers=headers)
        rp.raise_for_status()    # 如果响应状态码不是 200，就主动抛出异常
        psoup = BeautifulSoup(rp.text, 'html5lib')  # html5lib为pythonista上的解析器
        #<p><label>Mobile 电话：</label><big>90382241</big></p>	<p><label>WeChat 微信：</label><big>meixi112528</big>
           
        pname = ''
        pphone = ''
        pwechat= ''
        pvideo = ''
        
        details = psoup.find('ul','details')
        divs  = details.find_all('li')
      
        for div in divs:
            
            att_value = div.text
            #print (att_value)
            if ('Name' in att_value):
                nindex = att_value.find('姓名')+2##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pname = att_value[nindex:] #截取第七个字符到结尾
                
            if ('Mobile' in att_value):
                nindex = att_value.find('电话')+2##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pphone = att_value[nindex:] #截取第七个字符到结尾
                  
            if ('WhatsApp' in att_value):
                nindex = att_value.find('WhatsApp')+8##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                pphone = att_value[nindex:] #截取第七个字符到结尾
       
        pname=pname.replace(' ','')
        pname=pname.strip()
        name_data.append(pname)    
        
        pphone=pphone.replace(' ','')
        pphone=pphone.strip()
        if (len(pphone)>6):
            pphone = '+65'+pphone
        phone_data.append(pphone)   
        
        wechat_data.append(pwechat)  
        
        try:
            pvideo = psoup.find('video')['src']
            if (len(pvideo)>0):
                pvideo = 'http://www.sgbense.com'+pvideo
        except (AttributeError,TypeError):
            pass
        
        video_data.append(pvideo)
        
        print (pname,pphone,pwechat)
        
except (AttributeError,TypeError):
    pass

time.sleep( 60)
print(len(urls_data),len(name_data),len(phone_data),len(wechat_data),len(video_data))
#------------------------------------------------------------------------------------

url='http://www.sg1011.com/'

r = requests.get(url, headers=headers)
r.raise_for_status() 
soup = BeautifulSoup(r.text, 'html5lib')  # html5lib为pythonista上的解析器

try:
    
    pics = soup.find_all('div','pics-box')
    for i in range(len(pics)):
         peoples = pics[i]  #全部 所有
        
         for people in peoples.find('ul').find_all('li'):
           
            purl  = people.find('a')['href']  
            purl=url+purl.replace("\n","")
            urls_data.append(purl)
            #另一个页面，电话与微信
            time.sleep( 1)
            
            rp = requests.get(purl, headers=headers)
            rp.raise_for_status()    # 如果响应状态码不是 200，就主动抛出异常
            psoup = BeautifulSoup(rp.text, 'html5lib')  # html5lib为pythonista上的解析器
            
            pname = ''
            pphone = ''
            pwechat= ''
            pvideo = ''
            
            about = psoup.find('div','about')
            divs  = psoup.find_all('p')
     
            for div in divs:
           
                att_value = div.text
               
                if ('Name' in att_value):
                    nindex = att_value.find(':')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                    pname = att_value[nindex:] #截取第七个字符到结尾
                    
                if ('Mobile' in att_value):
                    nindex = att_value.find('：')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                    pphone = att_value[nindex:] #截取第七个字符到结尾
                 
            pname=pname.replace(' ','')
            pname=pname.strip()
            name_data.append(pname)    
        
            pphone=pphone.replace(' ','')
            pphone=pphone.strip()
            if (len(pphone)>6):
                pphone = '+65'+pphone
            phone_data.append(pphone)   
            
            wechat_data.append(pwechat)  
            
            try:
                pvideo = psoup.find('video')['src']
                if (len(pvideo)>0):
                    pvideo = 'http://www.sgbense.com'+pvideo
            except (AttributeError,TypeError):
                pass
        
            video_data.append(pvideo)
            
            print (pname,pphone)
            
except (AttributeError,TypeError):
    pass

time.sleep( 60)
print(len(urls_data),len(name_data),len(phone_data),len(wechat_data),len(video_data))
#------------------------------------------------------------------------------------

url='http://www.momosex.net/'

r = requests.get(url, headers=headers)
r.raise_for_status() 
soup = BeautifulSoup(r.text, 'html5lib')  # html5lib为pythonista上的解析器

try:
    
    pics = soup.find_all('div','pics-box')
    for i in range(len(pics)):
         peoples = pics[i]  #全部 头版 普通
        
         for people in peoples.find('ul').find_all('li'):
           
            purl  = people.find('a')['href']  
            purl=url+purl.replace("\n","")
            time.sleep( 1)
            #另一个页面，电话与微信
            urls_data.append(purl)
            rp = requests.get(purl, headers=headers)
            rp.raise_for_status()    # 如果响应状态码不是 200，就主动抛出异常
            psoup = BeautifulSoup(rp.text, 'html5lib')  # html5lib为pythonista上的解析器
            
            pname = ''
            pphone = ''
            pwechat= ''
            pvideo = ''
            
            about = psoup.find('div','about')
            divs  = psoup.find_all('p')
     
            pphone = about.find('div','contact').find('a').text # tel # sms #wechat
            print (pphone)
            for div in divs:
           
                att_value = div.text
                
                #<div class="contact">
			   #<a href="tel:81549543" class="tel">8154 9543</a>
                
                if ('Name' in att_value):
                    nindex = att_value.find('：')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                    pname = att_value[nindex:] #截取第七个字符到结尾
                    
            pname=pname.replace(' ','')
            pname=pname.strip()
            name_data.append(pname)    
        
            pphone=pphone.replace(' ','')
            pphone=pphone.strip()
            if (len(pphone)>6):
                pphone = '+65'+pphone
            phone_data.append(pphone)  
               
            wechat_data.append(pwechat)  
            try:
                pvideo = psoup.find('video')['src']
                if (len(pvideo)>0):
                    pvideo = 'http://www.momosex.net'+pvideo
            except (AttributeError,TypeError):
                pass
        
            video_data.append(pvideo)
            
            print (pname,pphone)
            
except (AttributeError,TypeError):
    pass       

time.sleep( 60)
#------------------------------------------------------------------------------------

url='http://8090girl.net/'  #狼网一个模版，可能是一家

r = requests.get(url, headers=headers)
r.raise_for_status() 
soup = BeautifulSoup(r.text, 'html5lib')  # html5lib为pythonista上的解析器

try:
    
    pics = soup.find_all('div','pics-box')
   
    for i in range(len(pics)):
         peoples = pics[i]  #全部 头版 普通
         for people in peoples.find('ul','fix').find_all('li'):
           
            purl  = people.find('a')['href']  
            purl=url+purl.replace("\n","")
            print (purl)
            
            time.sleep( 1)
            #另一个页面，电话与微信
            urls_data.append(purl)
            rp = requests.get(purl, headers=headers)
            rp.raise_for_status()    # 如果响应状态码不是 200，就主动抛出异常
            psoup = BeautifulSoup(rp.text, 'html5lib')  # html5lib为pythonista上的解析器
            
            pname = ''
            pphone = ''
            pwechat = ''
            pvideo = ''
             
            about = psoup.find('div','about')
            divs  = psoup.find_all('p')
     
            for div in divs:
           
                att_value = div.text
                
                #<div class="contact">
			   #<a href="tel:81549543" class="tel">8154 9543</a>
                
                if ('Name' in att_value):
                    nindex = att_value.find('：')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                    pname = att_value[nindex:] #截取第七个字符到结尾
                if ('Wechat' in att_value):
                    nindex = att_value.find('：')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                    pwechat = att_value[nindex:] #截取第七个字符到结尾  
                if ('Mobile' in att_value):
                    nindex = att_value.find(':')+1##从下标0开始，查找在字符串里第一个出现的子串，返回结果：0
                    pphone = att_value[nindex:] #截取第七个字符到结尾   
                    
            pname=pname.replace(' ','')
            pname=pname.strip()
            name_data.append(pname)    
        
            pphone=pphone.replace(' ','')
            pphone=pphone.strip()
            if (len(pphone)>6):
                pphone = '+65'+pphone
            phone_data.append(pphone)  
               
            wechat_data.append(pwechat)  
            try:
                pvideo = psoup.find('video')['src']
                if (len(pvideo)>0):
                    pvideo = 'http://www.8090girl.net'+pvideo
            except (AttributeError,TypeError):
                pass
        
            video_data.append(pvideo)
            
            print (pname,pphone,pwechat)
            
except (AttributeError,TypeError):
    pass

print(len(urls_data),len(name_data),len(phone_data),len(wechat_data),len(video_data))

table=pd.DataFrame({'name':name_data,'phone':phone_data,'wechat':wechat_data,'url':urls_data,'video':video_data})

date = time.strftime('%Y%m%d',time.localtime(time.time()))

table.to_csv(r'sg/girls%s.csv' % (date),encoding = "utf_8_sig")
